package org.carrot2.examples.clustering;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.StringTokenizer;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

public class URLConnectionReader {
	public String query;
	public int hits;
	public ArrayList<ArrayList<String>> docArray;
	public String url;
	public boolean intent;
	public HashMap<String,String> thumbToUrl;
	public boolean timeQuery;
	public long months;
	public long days;
	public long minutes;
	
	
	public URLConnectionReader(String query,int abc){
		this.query=query;
		docArray = new ArrayList<ArrayList<String>>();
		hits=abc;
		thumbToUrl=new HashMap<String,String>();
	}

	public URLConnectionReader(String temp, int abc,boolean intent) {
		query = temp;
		hits = abc;
		docArray = new ArrayList<ArrayList<String>>();
		url=new String();
		this.intent =intent;
		thumbToUrl=new HashMap<String,String>();
		this.months=0;
		this.days=0;
		this.minutes=0;
		
	}


	public ArrayList<ArrayList<String>> xmlToDocArrayTime(long days,long hours,long minutes) throws Exception {
		
		
		long currTime=System.currentTimeMillis()/1000;
		long val=(((days)*24+hours)*60+minutes)*60;
		System.out.println(days+"   "+hours+"  "+minutes);
		long oldTime=currTime-val;
		
		
		System.out.println("query  "+this.query);
		System.out.println("abcdef");
		
		
		if(this.query.compareTo("")==0){
						
			url="http://pixqr01.img.search.bf1.yahoo.com:4080/?restrict=pix&query=lang:en lastvespafeedtime:["+oldTime+";"+currTime+"]&hits=400&template=XMLImage&thumbsize=200%2C500%2C1024";
			
		}
		else{
			url="http://pixqr01.img.search.bf1.yahoo.com:4080/?restrict=pix&query="+this.query+" lastvespafeedtime:["+oldTime+";"+currTime+"]&hits=400&template=XMLImage&thumbsize=200%2C500%2C1024";	
		}

		
		
		String data = "";

		
		if(query!=null)query=query.replace(' ','+');
		
		int urlIndex=0;

		while(urlIndex<(hits/400)){
			
			
			String temp1=url.replace(" ", "%20");
			temp1=temp1+"&offset="+(urlIndex*400);
			System.out.println(temp1);
			URL oracle = new URL(temp1);
			urlIndex++;
			HttpURLConnection connection = (HttpURLConnection) oracle
			.openConnection();
			connection.setRequestMethod("GET");
			connection.setRequestProperty("Content-Type", "text/xml");
			

			BufferedReader in = new BufferedReader(new InputStreamReader(
			connection.getInputStream()));
			String inputLine;
			StringBuffer sb = new StringBuffer();
			
			
			while ((inputLine = in.readLine()) != null)
				sb.append(inputLine);		
			
			in.close();
			System.out.println(sb.length());
			data=sb.toString();
			System.out.println(data.length());
			
			try {
				
				DocumentBuilderFactory dbFactory = DocumentBuilderFactory
				.newInstance();
				DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
				Document doc = loadXMLFromString(data);
				doc.getDocumentElement().normalize();
				NodeList nList = doc.getElementsByTagName("hit");
				
				for (int temp = 0; temp < nList.getLength(); temp++) {
				
					Node nNode = nList.item(temp);
		
			/*			***** OLD TEMPLATE PARSING  ************
						
							if (nNode.getNodeType() == Node.ELEMENT_NODE) {
								Element eElement = (Element) nNode;
								
								ArrayList<String> docDesc = new ArrayList<String>(); 
								String thumbnails=getAtributeValue("thumbnails", eElement);
								
								
								
								
								String thumbURL= "http://d.yimg.com/sr/"+thumbnails.substring(thumbnails.indexOf("<url>")+5,thumbnails.indexOf("</url>"));
								
								NodeList n3=eElement.getElementsByTagName("ImageUrl");
								Element E1=(Element)n3.item(0);
								String IMGURL=E1.getFirstChild().getNodeValue();
								
								thumbToUrl.put(thumbURL, IMGURL);
								
								
								docDesc.add(thumbURL);
								
								if(on.compareTo("d")==0){
								
								docDesc.add(getAtributeValue1("capentities", eElement));
								docDesc.add(getAtributeValue("desc", eElement));
								
							}
							else if(on.compareTo("c")==0)
							{
								docDesc.add(getAtributeValue("desc", eElement));
								docDesc.add(getAtributeValue1("capentities", eElement));	
							}
							else
							{
								docDesc.add(getAtributeValue1("conceptkeywords", eElement));
								String t1=getAtributeValue("title", eElement);
								
		//						/t1=t1.concat("::"+getAtributeValue("ccmid", eElement));
								
								docDesc.add(t1);
							
							}
							
							docArray.add(docDesc);
							}
						}
					} catch (Exception e) {
					e.printStackTrace();
					}
					}*/
					
					
					/****** NEW TEMPLATE PARSING *************/
					if (nNode.getNodeType() == Node.ELEMENT_NODE) {
						Element eElement = (Element) nNode;
					
						ArrayList<String> docDesc = new ArrayList<String>();
						
						NodeList n2=eElement.getElementsByTagName("Url");
						Element E=(Element)n2.item(0);
						String thumbURL=E.getFirstChild().getNodeValue();
						
						docDesc.add(thumbURL);
						
						NodeList n3=eElement.getElementsByTagName("ImageUrl");
						Element E1=(Element)n3.item(0);
						String IMGURL=E1.getFirstChild().getNodeValue();
						
						thumbToUrl.put(thumbURL, IMGURL);
						
						
						
						
								
								String tempCK=getAtributeValue2("ConceptKeywords", eElement);
								
								docDesc.add(tempCK);
								
								NodeList titles=eElement.getElementsByTagName("Title");								
								String title=titles.item(0).getTextContent().toString();								
								docDesc.add(title);
								
							
						
							
							docArray.add(docDesc);
							}
						}
					} catch (Exception e) {
					e.printStackTrace();
					}
					
				
				System.out.println("time range query fetch");
			}
		

		
		
		
		
		
		return docArray;
	}
	public ArrayList<ArrayList<String>> xmlToDocArray(String on, String source, int intentLimit) throws Exception { 
				
		String data = "";
		int index=0;
		
		query=query.replace(' ','+');
		
		if(intent){
		
			
			if(source.compareTo("getty")==0){
				url="http://ddqrs.img.search.vip.sp2.yahoo.com:4080/?template=XMLImage&hq=on&hits=100&region=us&thumbsize=200%2C500%2C1024&l=en&stype=filter&filtertype=premium&custid=imagesearch%20partnername:getty&query="+query;
				//url="http://pixqr01.img.search.bf1.yahoo.com:4080/?query=("+query+")+AND+("+tempList+")%20partnername:getty&hits=400&restrict=pix&type=adv";				
			}
			else
			if(source.compareTo("ono")==0){
				url="http://ddqrs.img.search.vip.sp2.yahoo.com:4080/?template=XMLImage&hq=on&hits=100&region=us&thumbsize=200%2C500%2C1024&l=en&stype=filter&filtertype=premium&custid=imagesearch%20-partnername:gettyquery="+query;
				//url="http://pixqr01.img.search.bf1.yahoo.com:4080/?query=("+query+")+AND+("+tempList+")%20-partnername:getty&hits=400&restrict=pix&type=adv";				
			}
			else{
				url="http://ddqrs.img.search.vip.sp2.yahoo.com:4080/?template=XMLImage&hq=on&hits=100&thumbsize=200%2C500%2C1024&l=en&stype=filter&filtertype=premium&custid=imagesearch&query="+query;				
			}	
		}
		
		else{
		
			if(source.compareTo("getty")==0){				
				url="http://pixqr01.img.search.bf1.yahoo.com:4080/?query="+query+"%20partnername:getty&hits=400&restrict=pix&template=XMLImage";				
			}
			else
			if(source.compareTo("ono")==0){
				url="http://pixqr01.img.search.bf1.yahoo.com:4080/?query="+query+"%20-partnername:getty&hits=400&restrict=pix&template=XMLImage";				
			}
			else{
				url="http://pixqr01.img.search.bf1.yahoo.com:4080/?query="+ query + "&hits=400&restrict=pix&template=XMLImage";				
			}
		}
		
		
		//System.out.println(url);
		int urlIndex=0;
		if(!intent){
		while(urlIndex<(hits/400)){
			

			String temp1=url+"&offset="+(urlIndex*400);
			System.out.println(temp1);
			URL oracle = new URL(temp1);
			urlIndex++;
			HttpURLConnection connection = (HttpURLConnection) oracle
			.openConnection();
			connection.setRequestMethod("GET");
			connection.setRequestProperty("Content-Type", "text/xml");
			

			BufferedReader in = new BufferedReader(new InputStreamReader(
			connection.getInputStream()));
			String inputLine;
			StringBuffer sb = new StringBuffer();
			
			
			while ((inputLine = in.readLine()) != null)
				sb.append(inputLine);		
			
			in.close();
			data=sb.toString();
			
			try {
				
				DocumentBuilderFactory dbFactory = DocumentBuilderFactory
				.newInstance();
				DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
				Document doc = loadXMLFromString(data);
				doc.getDocumentElement().normalize();
				NodeList nList = doc.getElementsByTagName("hit");
				
				for (int temp = 0; temp < nList.getLength(); temp++) {
				
					Node nNode = nList.item(temp);
		
			/*			***** OLD TEMPLATE PARSING  ************
						
							if (nNode.getNodeType() == Node.ELEMENT_NODE) {
								Element eElement = (Element) nNode;
								
								ArrayList<String> docDesc = new ArrayList<String>(); 
								String thumbnails=getAtributeValue("thumbnails", eElement);
								
								
								
								
								String thumbURL= "http://d.yimg.com/sr/"+thumbnails.substring(thumbnails.indexOf("<url>")+5,thumbnails.indexOf("</url>"));
								
								NodeList n3=eElement.getElementsByTagName("ImageUrl");
								Element E1=(Element)n3.item(0);
								String IMGURL=E1.getFirstChild().getNodeValue();
								
								thumbToUrl.put(thumbURL, IMGURL);
								
								
								docDesc.add(thumbURL);
								
								if(on.compareTo("d")==0){
								
								docDesc.add(getAtributeValue1("capentities", eElement));
								docDesc.add(getAtributeValue("desc", eElement));
								
							}
							else if(on.compareTo("c")==0)
							{
								docDesc.add(getAtributeValue("desc", eElement));
								docDesc.add(getAtributeValue1("capentities", eElement));	
							}
							else
							{
								docDesc.add(getAtributeValue1("conceptkeywords", eElement));
								String t1=getAtributeValue("title", eElement);
								
		//						/t1=t1.concat("::"+getAtributeValue("ccmid", eElement));
								
								docDesc.add(t1);
							
							}
							
							docArray.add(docDesc);
							}
						}
					} catch (Exception e) {
					e.printStackTrace();
					}
					}*/
					
					
					/****** NEW TEMPLATE PARSING *************/
					if (nNode.getNodeType() == Node.ELEMENT_NODE) {
						Element eElement = (Element) nNode;
					
						ArrayList<String> docDesc = new ArrayList<String>();
						
						NodeList n2=eElement.getElementsByTagName("Url");
						Element E=(Element)n2.item(0);
						String thumbURL=E.getFirstChild().getNodeValue();
						
						docDesc.add(thumbURL);
						
						NodeList n3=eElement.getElementsByTagName("ImageUrl");
						Element E1=(Element)n3.item(0);
						String IMGURL=E1.getFirstChild().getNodeValue();
						
						thumbToUrl.put(thumbURL, IMGURL);
						
						
						if(on.compareTo("d")==0){
							
								docDesc.add(getAtributeValue1("capentities", eElement));
								docDesc.add(getAtributeValue("desc", eElement));
								
							}
						
							else if(on.compareTo("c")==0)
							{
								docDesc.add(getAtributeValue("desc", eElement));
								docDesc.add(getAtributeValue1("capentities", eElement));	
							}
							else
							{
								
								String tempCK=getAtributeValue2("ConceptKeywords", eElement);
								
								docDesc.add(tempCK);
								
								NodeList titles=eElement.getElementsByTagName("Title");								
								String title=titles.item(0).getTextContent().toString();								
								docDesc.add(title);
								
							
							}
							
							docArray.add(docDesc);
							}
						}
					} catch (Exception e) {
					e.printStackTrace();
					}
					
				
				System.out.println("Normal Images Fetch");
			}
		}
			else
			{
				while(urlIndex<(hits/100)){
					

					String temp1=url+"&offset="+(urlIndex*100);
				
					
					URL oracle = new URL(temp1);
					urlIndex++;
					HttpURLConnection connection = (HttpURLConnection) oracle
					.openConnection();
					connection.setRequestMethod("GET");
					connection.setRequestProperty("Content-Type", "text/xml");
					

					BufferedReader in = new BufferedReader(new InputStreamReader(
					connection.getInputStream()));
					String inputLine;
					StringBuffer sb = new StringBuffer();
					
					
					while ((inputLine = in.readLine()) != null)
						sb.append(inputLine);		
					
					in.close();
					data=sb.toString();
				
				try {
					
					DocumentBuilderFactory dbFactory = DocumentBuilderFactory
					.newInstance();
					DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
					Document doc = loadXMLFromString(data);
					doc.getDocumentElement().normalize();
					NodeList nList = doc.getElementsByTagName("Result");
					
					
					
					for (int temp = 0; temp < nList.getLength(); temp++) {
						
						Node nNode = nList.item(temp);
						if (nNode.getNodeType() == Node.ELEMENT_NODE) {
						Element eElement = (Element) nNode;
					
						ArrayList<String> docDesc = new ArrayList<String>();
						
						NodeList n2=eElement.getElementsByTagName("Url");
						Element E=(Element)n2.item(0);
						String thumbURL=E.getFirstChild().getNodeValue();
						
						docDesc.add(thumbURL);
						
						NodeList n3=eElement.getElementsByTagName("ImageUrl");
						Element E1=(Element)n3.item(0);
						String IMGURL=E1.getFirstChild().getNodeValue();
						
						thumbToUrl.put(thumbURL, IMGURL);
						
						
						if(on.compareTo("d")==0){
							
								docDesc.add(getAtributeValue1("capentities", eElement));
								docDesc.add(getAtributeValue("desc", eElement));
								
							}
						
							else if(on.compareTo("c")==0)
							{
								docDesc.add(getAtributeValue("desc", eElement));
								docDesc.add(getAtributeValue1("capentities", eElement));	
							}
							else
							{
								
								String tempCK=getAtributeValue2("ConceptKeywords", eElement);
								
								docDesc.add(tempCK);
								
								NodeList titles=eElement.getElementsByTagName("Title");								
								String title=titles.item(0).getTextContent().toString();								
								docDesc.add(title);
								
							
							}
							
							docArray.add(docDesc);
							}
						}
					} catch (Exception e) {
					e.printStackTrace();
					}
					
				
				System.out.println("intent based boosting requierd");
			}
		}
		return docArray;
}


	
	
	public static Document loadXMLFromString(String xml) throws Exception {
		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		DocumentBuilder builder = factory.newDocumentBuilder();
		InputSource is = new InputSource(new StringReader(xml));
		return builder.parse(is);
	}

	private static String getAtributeValue(String sTag, Element eElement) {
		String out = "";
		StringBuffer bf=new StringBuffer();
		NodeList links = eElement.getElementsByTagName("field");
		for (int i = 0; i < links.getLength(); i++) {
			Element link = (Element) links.item(i);
			String abc = link.getAttribute("name");
			if (sTag.compareTo(abc) == 0) {
				out = link.getFirstChild().getNodeValue();
			}
			
		}
		
		if(sTag.compareTo("desc")==0){
			
			StringTokenizer abc=new StringTokenizer(out);
			String temp;
			
			while(abc.hasMoreTokens()){
				
				temp=abc.nextToken();
				
				if(temp.compareTo("AFP")!=0 && temp.compareTo("(Photo")!=0)
					bf.append(temp+' ');
				else
					break;
			}
			String abcd=bf.toString();			
			return abcd;
		}
		else
			return out;
	}
	
	private static String getAtributeValue1(String sTag, Element eElement) {
		StringBuffer out = new StringBuffer();
		NodeList links = eElement.getElementsByTagName("field");
		for (int i = 0; i < links.getLength(); i++) {
			Element link = (Element) links.item(i);
			String abc = link.getAttribute("name");
			if (sTag.compareTo(abc) == 0) {	
				NodeList nList = link.getElementsByTagName("item");
				for (int temp = 0; temp < nList.getLength(); temp++) {
					Node nNode = nList.item(temp);
		out.append(nNode.getTextContent().toLowerCase().replace(' ', '_')+" ");
				}
			}
		}
		return out.toString();
	}
	
	/*
	 * for the new XML Format 
	 */
	private static String getAtributeValue2(String sTag, Element eElement) {
		StringBuffer out = new StringBuffer();
		NodeList links = eElement.getElementsByTagName(sTag);
		for (int i = 0; i < links.getLength(); i++) {
			Element link = (Element) links.item(i);
			NodeList nList = link.getElementsByTagName("item");
			
			for (int temp = 0; temp < nList.getLength(); temp++) {
				
					Node nNode = nList.item(temp);
					out.append(nNode.getTextContent().toLowerCase().replace(' ', '_')+" ");
				}
			
		}
		return out.toString();
	}
	
	
}